from pyquery import PyQuery as pq
import urllib.parse
from module import Function

class Parser(object):
    def paser(self, page_url, html_cont, basedir):
        doc = pq(html_cont)
        doc('.navbar-nav').remove()
        doc('a.logo').attr('href','#')
        doc('form.hidden-xs').remove()
        doc('.footer-content p').remove()
        doc('.sidebar').remove()
        doc('.content').css('padding-left','0').css('padding-right','0')
        doc('#ArticleRelation').remove()
        doc('#loadNextPageBtn').remove()
        doc('#pageFooter').remove()
        doc('#share-to').remove()
        doc('.footer-content').remove()
        html_cont = doc.outer_html()
        doc = pq(html_cont)
        new_urls = set()
        links = doc('.content a').items()
        for link in links:
            new_url = link.attr.href
            if len(new_url) < 5:
                continue
            check = False
            if new_url.find('http') == -1 and new_url.find('.') > -1:
                check = True
            elif new_url.find('tv.81.cn') > -1:
                check = True
            if check == True:
                new_full_url = urllib.parse.urljoin(page_url, new_url)
                new_urls.add(new_full_url)

        html_cont = html_cont.replace('http://tv.81.cn','/81zt')
        Function.savehtml(page_url,basedir,html_cont)

        return new_urls,html_cont
